Load Data

# devtools::install_github("ropensci/plotly")
library(plotly)
Loading required package: ggplot2
Registered S3 method overwritten by 'dplyr':
  method           from
  print.rowwise_df     
Need help? Try Stackoverflow: https://stackoverflow.com/tags/ggplot2.
Registered S3 method overwritten by 'data.table':
  method           from
  print.data.table     
Registered S3 methods overwritten by 'htmltools':
  method               from         
  print.html           tools:rstudio
  print.shiny.tag      tools:rstudio
  print.shiny.tag.list tools:rstudio
Registered S3 method overwritten by 'htmlwidgets':
  method           from         
  print.htmlwidget tools:rstudio

Attaching package: 㤼㸱plotly㤼㸲

The following object is masked from 㤼㸱package:ggplot2㤼㸲:

    last_plot

The following object is masked from 㤼㸱package:stats㤼㸲:

    filter

The following object is masked from 㤼㸱package:graphics㤼㸲:

    layout
library(DAAG)
Loading required package: lattice
library(ggplot2)
head(ais)

Normalize Data

min_max <- function(x) {
  return ((x - min(x))/(max(x) - min(x)))
}

x_train <- apply(ais[, 1:11], 2, min_max)

head(x_train)
         rcc       wcc         hc         hg       ferr       bmi       ssf    pcBfat
1 0.05479452 0.3818182 0.06722689 0.09210526 0.23008850 0.2156197 0.4693287 0.4723988
2 0.20890411 0.4545455 0.09663866 0.14473684 0.26548673 0.2218449 0.4328704 0.5242556
3 0.11643836 0.1545455 0.02100840 0.00000000 0.05752212 0.2891907 0.4432870 0.4767481
4 0.10616438 0.1818182 0.05882353 0.13157895 0.26991150 0.2903226 0.5694444 0.6032118
5 0.22260274 0.3181818 0.23529412 0.31578947 0.09292035 0.1250707 0.3026620 0.4018066
6 0.10273973 0.1000000 0.06302521 0.11842105 0.15044248 0.2427844 0.2731481 0.3328873
        lbm        ht        wt
1 0.4042434 0.7768595 0.4812646
2 0.3376605 0.6743802 0.4285714
3 0.2931323 0.4776860 0.3665105
4 0.3185371 0.5966942 0.4344262
5 0.2629816 0.5900826 0.3138173
6 0.2709380 0.4148760 0.3032787

PCA for Dimensional Reduction

pca <- prcomp(x_train)

pca
Standard deviations (1, .., p=11):
 [1] 0.385868910 0.289535274 0.197430062 0.175422597 0.148272925 0.111117183 0.053473287
 [8] 0.033736286 0.028849470 0.012240374 0.005661196

Rotation (n x k) = (11 x 11):
               PC1         PC2         PC3          PC4         PC5         PC6
rcc     0.33739235 -0.05838215  0.02150914 -0.358680602  0.21131404 -0.23211863
wcc     0.05689837  0.14962256 -0.19771247 -0.532764570 -0.79752803  0.12273024
hc      0.34607222 -0.06596026  0.02526416 -0.341668220  0.19414008 -0.15364551
hg      0.40981431 -0.05733335 -0.01043262 -0.356736075  0.26573951 -0.05898774
ferr    0.24216199  0.10387340 -0.89611758  0.300730010  0.02856695 -0.18827728
bmi     0.20640664  0.41353060 -0.01678115 -0.001072319  0.18943822  0.61774042
ssf    -0.24981756  0.53068491 -0.03108384 -0.168754158  0.17155065 -0.20819204
pcBfat -0.34297756  0.51318779 -0.03458144 -0.212507756  0.18630827 -0.23411251
lbm     0.40746613  0.23293517  0.18540244  0.255736225 -0.11865756  0.15393803
ht      0.25609714  0.20005781  0.30599704  0.289957285 -0.30572044 -0.59795919
wt      0.28690117  0.38144384  0.16193012  0.173086378 -0.05098682  0.08669181
                PC7         PC8          PC9          PC10          PC11
rcc     0.764342861  0.18766338 -0.188558338  0.0108028781 -0.0015251304
wcc    -0.006124073  0.01459060 -0.008900196 -0.0017048843  0.0010849554
hc     -0.135781504 -0.66536764  0.485478052 -0.0191951506 -0.0015475224
hg     -0.614251195  0.41548964 -0.278004128 -0.0072450005  0.0048633771
ferr    0.007387012 -0.02722995 -0.006486023  0.0002535866 -0.0002313937
bmi     0.078145496 -0.08355718 -0.040218913  0.5875136360  0.0934749543
ssf     0.002470876  0.43873186  0.599635163 -0.0254040883  0.0323658233
pcBfat -0.065008567 -0.38168309 -0.529387128 -0.1636377027  0.1618446920
lbm     0.044556113  0.03621892  0.051215815 -0.5186314003  0.6060216964
ht     -0.085340672 -0.01761235 -0.062260221  0.4996783929  0.0671823933
wt      0.018427026 -0.04136346 -0.058474104 -0.3289489617 -0.7695555910
qplot(x = 1:11, y = cumsum(pca$sdev)/sum(pca$sdev), geom="line")

As shown above, the first 6 components can explain 90% variance of the data.

Below shows how the first 2 principle components seperates male and female:

ggplot(data.frame(PC1=as.data.frame(pca$x)$PC1, PC2=as.data.frame(pca$x)$PC2), 
       aes(x = PC1, y = PC2, col = ais$sex)) + geom_point()

Below shows how the first 3 principle components seperates male and female:

# devtools::install_github("ropensci/plotly")
library(plotly)
Registered S3 method overwritten by 'data.table':
  method           from
  print.data.table     
Registered S3 method overwritten by 'htmlwidgets':
  method           from         
  print.htmlwidget tools:rstudio

Attaching package: 㤼㸱plotly㤼㸲

The following object is masked from 㤼㸱package:ggplot2㤼㸲:

    last_plot

The following object is masked from 㤼㸱package:stats㤼㸲:

    filter

The following object is masked from 㤼㸱package:graphics㤼㸲:

    layout
pca_plotly <- plot_ly(as.data.frame(pca$x), x = ~PC1, y = ~PC2, z = ~PC3, color = ~ais$sex) %>% add_markers()

pca_plotly
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels

AutoEncoder for Dimensional Reduction

# devtools::install_github("rstudio/tensorflow")  # make sure you have installed devtools
library("tensorflow")
library("keras")
# I have already installed keras with tensorflow backend through Conda, so no need to install again here
# reticulate::py_discover_config()  # this will show available installed tensorflow in python
keras::is_keras_available()  # If this reurns True, it means you Keras with tensorflow is available
[1] TRUE
head(x_train)
         rcc       wcc         hc         hg       ferr       bmi       ssf    pcBfat       lbm        ht
1 0.05479452 0.3818182 0.06722689 0.09210526 0.23008850 0.2156197 0.4693287 0.4723988 0.4042434 0.7768595
2 0.20890411 0.4545455 0.09663866 0.14473684 0.26548673 0.2218449 0.4328704 0.5242556 0.3376605 0.6743802
3 0.11643836 0.1545455 0.02100840 0.00000000 0.05752212 0.2891907 0.4432870 0.4767481 0.2931323 0.4776860
4 0.10616438 0.1818182 0.05882353 0.13157895 0.26991150 0.2903226 0.5694444 0.6032118 0.3185371 0.5966942
5 0.22260274 0.3181818 0.23529412 0.31578947 0.09292035 0.1250707 0.3026620 0.4018066 0.2629816 0.5900826
6 0.10273973 0.1000000 0.06302521 0.11842105 0.15044248 0.2427844 0.2731481 0.3328873 0.2709380 0.4148760
         wt
1 0.4812646
2 0.4285714
3 0.3665105
4 0.4344262
5 0.3138173
6 0.3032787
x_train <- as.matrix(x_train)

head(x_train)
         rcc       wcc         hc         hg       ferr       bmi       ssf    pcBfat
1 0.05479452 0.3818182 0.06722689 0.09210526 0.23008850 0.2156197 0.4693287 0.4723988
2 0.20890411 0.4545455 0.09663866 0.14473684 0.26548673 0.2218449 0.4328704 0.5242556
3 0.11643836 0.1545455 0.02100840 0.00000000 0.05752212 0.2891907 0.4432870 0.4767481
4 0.10616438 0.1818182 0.05882353 0.13157895 0.26991150 0.2903226 0.5694444 0.6032118
5 0.22260274 0.3181818 0.23529412 0.31578947 0.09292035 0.1250707 0.3026620 0.4018066
6 0.10273973 0.1000000 0.06302521 0.11842105 0.15044248 0.2427844 0.2731481 0.3328873
        lbm        ht        wt
1 0.4042434 0.7768595 0.4812646
2 0.3376605 0.6743802 0.4285714
3 0.2931323 0.4776860 0.3665105
4 0.3185371 0.5966942 0.4344262
5 0.2629816 0.5900826 0.3138173
6 0.2709380 0.4148760 0.3032787

Try 2 dimensions in the encoded layer

model <- keras_model_sequential()

model %>%
  layer_dense(units=6, activation="tanh",
              input_shape = ncol(x_train)) %>%
  layer_dense(units=2, activation="tanh", name="bottleneck") %>%
  layer_dense(units=6, activation="tanh") %>%
  layer_dense(units=ncol(x_train))
2019-11-28 21:26:32.725655: I tensorflow/core/platform/cpu_feature_guard.cc:145] This TensorFlow binary is optimized with Intel(R) MKL-DNN to use the following CPU instructions in performance critical operations:  AVX AVX2
To enable them in non-MKL-DNN operations, rebuild TensorFlow with the appropriate compiler flags.
2019-11-28 21:26:32.735363: I tensorflow/core/common_runtime/process_util.cc:115] Creating new thread pool with default inter op setting: 8. Tune using inter_op_parallelism_threads for best performance.
summary(model)
Model: "sequential"
___________________________________________________________________________________________
Layer (type)                            Output Shape                         Param #       
===========================================================================================
dense (Dense)                           (None, 6)                            72            
___________________________________________________________________________________________
bottleneck (Dense)                      (None, 2)                            14            
___________________________________________________________________________________________
dense_1 (Dense)                         (None, 6)                            18            
___________________________________________________________________________________________
dense_2 (Dense)                         (None, 11)                           77            
===========================================================================================
Total params: 181
Trainable params: 181
Non-trainable params: 0
___________________________________________________________________________________________
model %>% compile(
  loss = "mean_squared_error",
  optimizer = "adam"
)
model %>% fit(
  x = x_train,
  y = x_train,  # here y is still x_train
  epochs = 2000,
  verbose = 0
)
mse.ae2 <- evaluate(model, x_train, x_train)

202/1 [========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================       loss 
====================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 0s 66us/sample - loss: 0.0126
0.009373661 
# extract the encoded layer (bottleneck layer)
intermediate_layer_model <- keras_model(inputs = model$input,
                                   outputs = get_layer(model, "bottleneck")$output)
intermediate_output <- predict(intermediate_layer_model, x_train)

Below shows how the encoded layer seperates male and female: To be honest, I didn’t see much difference here, comparing with PCA method above….

ggplot(data.frame(PC1 = intermediate_output[,1], PC2 = intermediate_output[,2]), aes(x = PC1, y = PC2, col = ais$sex)) + geom_point()

Try 3 dimensions in the encoded layer

model2 <- keras_model_sequential()

model2 %>%
  layer_dense(units=6, activation="tanh",
              input_shape = ncol(x_train)) %>%
  layer_dense(units=3, activation="tanh", name="bottleneck") %>%
  layer_dense(units=6, activation="tanh") %>%
  layer_dense(units=ncol(x_train))
summary(model2)
___________________________________________________________________________________________
Layer (type)                            Output Shape                         Param #       
===========================================================================================
dense_3 (Dense)                         (None, 6)                            72            
___________________________________________________________________________________________
bottleneck (Dense)                      (None, 3)                            21            
___________________________________________________________________________________________
dense_4 (Dense)                         (None, 6)                            24            
___________________________________________________________________________________________
dense_5 (Dense)                         (None, 11)                           77            
===========================================================================================
Total params: 194
Trainable params: 194
Non-trainable params: 0
___________________________________________________________________________________________
# compile model
model2 %>% compile(
  loss = "mean_squared_error", 
  optimizer = "adam"
)

# fit model
model2 %>% fit(
  x = x_train, 
  y = x_train, 
  epochs = 2000,
  verbose = 0
)

# evaluate the model
evaluate(model2, x_train, x_train)

 32/202 [===>..........................] - ETA: 1s
202/202 [==============================] - 0s 973us/step
       loss 
0.006340118 

It seems that it does seperate better than PCA above

intermediate_layer_model <- keras_model(inputs = model2$input, outputs = get_layer(model2, "bottleneck")$output)
intermediate_output <- predict(intermediate_layer_model, x_train)

# plot the reduced dat set
aedf <- data.frame(node1 = intermediate_output[,1], node2 = intermediate_output[,2], node3 = intermediate_output[,3])
ae_plotly <- plot_ly(aedf, x = ~node1, y = ~node2, z = ~node3, color = ~ais$sex) %>% add_markers()

ae_plotly
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels

Reconstruction Error: PCA vs AutoEncoder

# pCA reconstruction
pca.recon <- function(pca, x, k){
  mu <- matrix(rep(pca$center, nrow(pca$x)), nrow = nrow(pca$x), byrow = T)
  recon <- pca$x[,1:k] %*% t(pca$rotation[,1:k]) + mu
  mse <- mean((recon - x)^2)
  return(list(x = recon, mse = mse))
}

xhat <- rep(NA, 10)
for(k in 1:10){
  xhat[k] <- pca.recon(pca, x_train, k)$mse
}
# Autoencoder reconstruction
# autoencoder last layer is reconstruction
ae.mse <- rep(NA, 5)
for(k in 1:5){
  modelk <- keras_model_sequential()
  modelk %>%
    layer_dense(units = 6, activation = "tanh", input_shape = ncol(x_train)) %>%
    layer_dense(units = k, activation = "tanh", name = "bottleneck") %>%
    layer_dense(units = 6, activation = "tanh") %>%
    layer_dense(units = ncol(x_train))

  modelk %>% compile(
    loss = "mean_squared_error", 
    optimizer = "adam"
  )

  modelk %>% fit(
    x = x_train, 
    y = x_train, 
    epochs = 5000,
    verbose = 0
  )

  ae.mse[k] <- unname(evaluate(modelk, x_train, x_train))
}

202/1 [============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 0s 885us/sample - loss: 0.0149
C:\Users\hanhan\ANACON~1\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.183592). Check your callbacks.

202/1 [============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 0s 971us/sample - loss: 0.0122

202/1 [============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 0s 991us/sample - loss: 0.0064

202/1 [============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 0s 902us/sample - loss: 0.0042

202/1 [============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 0s 825us/sample - loss: 0.0016
df <- data.frame(k = c(1:10, 1:5), mse = c(xhat, ae.mse), method = c(rep("pca", 10), rep("autoencoder", 5)))
ggplot(df, aes(x = k, y = mse, col = method)) + geom_line()

Observation

  • When k is small, autoencoder has less reconstruction error
  • When k is larger, autoencoder and PCA apepar to have almost the same reconstruction error
LS0tDQp0aXRsZTogIkRpbWVuc2lvbmFsIFJlZHVjdGlvbiAtIEF1dG9FbmNvZGVyIHZzIFBDQSINCm91dHB1dDoNCiAgaHRtbF9ub3RlYm9vazogZGVmYXVsdA0KICBwZGZfZG9jdW1lbnQ6IGRlZmF1bHQNCi0tLQ0KDQojIyMgTG9hZCBEYXRhDQpgYGB7cn0NCiMgZGV2dG9vbHM6Omluc3RhbGxfZ2l0aHViKCJyb3BlbnNjaS9wbG90bHkiKQ0KbGlicmFyeShwbG90bHkpDQpsaWJyYXJ5KERBQUcpDQpsaWJyYXJ5KGdncGxvdDIpDQpoZWFkKGFpcykNCmBgYA0KDQoNCiMjIyBOb3JtYWxpemUgRGF0YQ0KYGBge3J9DQptaW5fbWF4IDwtIGZ1bmN0aW9uKHgpIHsNCiAgcmV0dXJuICgoeCAtIG1pbih4KSkvKG1heCh4KSAtIG1pbih4KSkpDQp9DQoNCnhfdHJhaW4gPC0gYXBwbHkoYWlzWywgMToxMV0sIDIsIG1pbl9tYXgpDQoNCmhlYWQoeF90cmFpbikNCmBgYA0KDQojIyMgUENBIGZvciBEaW1lbnNpb25hbCBSZWR1Y3Rpb24NCmBgYHtyfQ0KcGNhIDwtIHByY29tcCh4X3RyYWluKQ0KDQpwY2ENCmBgYA0KDQpgYGB7cn0NCnFwbG90KHggPSAxOjExLCB5ID0gY3Vtc3VtKHBjYSRzZGV2KS9zdW0ocGNhJHNkZXYpLCBnZW9tPSJsaW5lIikNCmBgYA0KQXMgc2hvd24gYWJvdmUsIHRoZSBmaXJzdCA2IGNvbXBvbmVudHMgY2FuIGV4cGxhaW4gOTAlIHZhcmlhbmNlIG9mIHRoZSBkYXRhLg0KDQpCZWxvdyBzaG93cyBob3cgdGhlIGZpcnN0IDIgcHJpbmNpcGxlIGNvbXBvbmVudHMgc2VwZXJhdGVzIG1hbGUgYW5kIGZlbWFsZToNCmBgYHtyfQ0KZ2dwbG90KGRhdGEuZnJhbWUoUEMxPWFzLmRhdGEuZnJhbWUocGNhJHgpJFBDMSwgUEMyPWFzLmRhdGEuZnJhbWUocGNhJHgpJFBDMiksIA0KICAgICAgIGFlcyh4ID0gUEMxLCB5ID0gUEMyLCBjb2wgPSBhaXMkc2V4KSkgKyBnZW9tX3BvaW50KCkNCmBgYA0KDQoNCg0KQmVsb3cgc2hvd3MgaG93IHRoZSBmaXJzdCAzIHByaW5jaXBsZSBjb21wb25lbnRzIHNlcGVyYXRlcyBtYWxlIGFuZCBmZW1hbGU6DQpgYGB7cn0NCnBjYV9wbG90bHkgPC0gcGxvdF9seShhcy5kYXRhLmZyYW1lKHBjYSR4KSwgeCA9IH5QQzEsIHkgPSB+UEMyLCB6ID0gflBDMywgY29sb3IgPSB+YWlzJHNleCkgJT4lIGFkZF9tYXJrZXJzKCkNCg0KcGNhX3Bsb3RseQ0KYGBgDQoNCg0KIyMjIEF1dG9FbmNvZGVyIGZvciBEaW1lbnNpb25hbCBSZWR1Y3Rpb24NCmBgYHtyfQ0KIyBkZXZ0b29sczo6aW5zdGFsbF9naXRodWIoInJzdHVkaW8vdGVuc29yZmxvdyIpICAjIG1ha2Ugc3VyZSB5b3UgaGF2ZSBpbnN0YWxsZWQgZGV2dG9vbHMNCmxpYnJhcnkoInRlbnNvcmZsb3ciKQ0KbGlicmFyeSgia2VyYXMiKQ0KYGBgDQoNCmBgYHtyfQ0KIyBJIGhhdmUgYWxyZWFkeSBpbnN0YWxsZWQga2VyYXMgd2l0aCB0ZW5zb3JmbG93IGJhY2tlbmQgdGhyb3VnaCBDb25kYSwgc28gbm8gbmVlZCB0byBpbnN0YWxsIGFnYWluIGhlcmUNCnJldGljdWxhdGU6OnB5X2Rpc2NvdmVyX2NvbmZpZygpICAjIHRoaXMgd2lsbCBzaG93IGF2YWlsYWJsZSBpbnN0YWxsZWQgdGVuc29yZmxvdyBpbiBweXRob24NCmtlcmFzOjppc19rZXJhc19hdmFpbGFibGUoKSAgIyBJZiB0aGlzIHJldXJucyBUcnVlLCBpdCBtZWFucyB5b3UgS2VyYXMgd2l0aCB0ZW5zb3JmbG93IGlzIGF2YWlsYWJsZQ0KYGBgDQoNCmBgYHtyfQ0KaGVhZCh4X3RyYWluKQ0KYGBgDQoNCmBgYHtyfQ0KeF90cmFpbiA8LSBhcy5tYXRyaXgoeF90cmFpbikNCg0KaGVhZCh4X3RyYWluKQ0KYGBgDQoNCiMjIyMgVHJ5IDIgZGltZW5zaW9ucyBpbiB0aGUgZW5jb2RlZCBsYXllcg0KYGBge3J9DQptb2RlbCA8LSBrZXJhc19tb2RlbF9zZXF1ZW50aWFsKCkNCg0KbW9kZWwgJT4lDQogIGxheWVyX2RlbnNlKHVuaXRzPTYsIGFjdGl2YXRpb249InRhbmgiLA0KICAgICAgICAgICAgICBpbnB1dF9zaGFwZSA9IG5jb2woeF90cmFpbikpICU+JQ0KICBsYXllcl9kZW5zZSh1bml0cz0yLCBhY3RpdmF0aW9uPSJ0YW5oIiwgbmFtZT0iYm90dGxlbmVjayIpICU+JQ0KICBsYXllcl9kZW5zZSh1bml0cz02LCBhY3RpdmF0aW9uPSJ0YW5oIikgJT4lDQogIGxheWVyX2RlbnNlKHVuaXRzPW5jb2woeF90cmFpbikpDQpgYGANCg0KDQpgYGB7cn0NCnN1bW1hcnkobW9kZWwpDQpgYGANCg0KYGBge3J9DQptb2RlbCAlPiUgY29tcGlsZSgNCiAgbG9zcyA9ICJtZWFuX3NxdWFyZWRfZXJyb3IiLA0KICBvcHRpbWl6ZXIgPSAiYWRhbSINCikNCmBgYA0KDQpgYGB7cn0NCm1vZGVsICU+JSBmaXQoDQogIHggPSB4X3RyYWluLA0KICB5ID0geF90cmFpbiwgICMgaGVyZSB5IGlzIHN0aWxsIHhfdHJhaW4NCiAgZXBvY2hzID0gMjAwMCwNCiAgdmVyYm9zZSA9IDANCikNCmBgYA0KDQpgYGB7cn0NCm1zZS5hZTIgPC0gZXZhbHVhdGUobW9kZWwsIHhfdHJhaW4sIHhfdHJhaW4pDQptc2UuYWUyDQpgYGANCg0KYGBge3J9DQojIGV4dHJhY3QgdGhlIGVuY29kZWQgbGF5ZXIgKGJvdHRsZW5lY2sgbGF5ZXIpDQppbnRlcm1lZGlhdGVfbGF5ZXJfbW9kZWwgPC0ga2VyYXNfbW9kZWwoaW5wdXRzID0gbW9kZWwkaW5wdXQsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG91dHB1dHMgPSBnZXRfbGF5ZXIobW9kZWwsICJib3R0bGVuZWNrIikkb3V0cHV0KQ0KaW50ZXJtZWRpYXRlX291dHB1dCA8LSBwcmVkaWN0KGludGVybWVkaWF0ZV9sYXllcl9tb2RlbCwgeF90cmFpbikNCmBgYA0KDQpCZWxvdyBzaG93cyBob3cgdGhlIGVuY29kZWQgbGF5ZXIgc2VwZXJhdGVzIG1hbGUgYW5kIGZlbWFsZToNClRvIGJlIGhvbmVzdCwgSSBkaWRuJ3Qgc2VlIG11Y2ggZGlmZmVyZW5jZSBoZXJlLCBjb21wYXJpbmcgd2l0aCBQQ0EgbWV0aG9kIGFib3ZlLi4uLg0KYGBge3J9DQpnZ3Bsb3QoZGF0YS5mcmFtZShQQzEgPSBpbnRlcm1lZGlhdGVfb3V0cHV0WywxXSwgUEMyID0gaW50ZXJtZWRpYXRlX291dHB1dFssMl0pLCANCiAgICAgICBhZXMoeCA9IFBDMSwgeSA9IFBDMiwgY29sID0gYWlzJHNleCkpICsgZ2VvbV9wb2ludCgpDQpgYGANCg0KIyMjIyBUcnkgMyBkaW1lbnNpb25zIGluIHRoZSBlbmNvZGVkIGxheWVyDQpgYGB7cn0NCm1vZGVsMiA8LSBrZXJhc19tb2RlbF9zZXF1ZW50aWFsKCkNCg0KbW9kZWwyICU+JQ0KICBsYXllcl9kZW5zZSh1bml0cz02LCBhY3RpdmF0aW9uPSJ0YW5oIiwNCiAgICAgICAgICAgICAgaW5wdXRfc2hhcGUgPSBuY29sKHhfdHJhaW4pKSAlPiUNCiAgbGF5ZXJfZGVuc2UodW5pdHM9MywgYWN0aXZhdGlvbj0idGFuaCIsIG5hbWU9ImJvdHRsZW5lY2siKSAlPiUNCiAgbGF5ZXJfZGVuc2UodW5pdHM9NiwgYWN0aXZhdGlvbj0idGFuaCIpICU+JQ0KICBsYXllcl9kZW5zZSh1bml0cz1uY29sKHhfdHJhaW4pKQ0KYGBgDQoNCmBgYHtyfQ0Kc3VtbWFyeShtb2RlbDIpDQpgYGANCg0KYGBge3J9DQojIGNvbXBpbGUgbW9kZWwNCm1vZGVsMiAlPiUgY29tcGlsZSgNCiAgbG9zcyA9ICJtZWFuX3NxdWFyZWRfZXJyb3IiLCANCiAgb3B0aW1pemVyID0gImFkYW0iDQopDQoNCiMgZml0IG1vZGVsDQptb2RlbDIgJT4lIGZpdCgNCiAgeCA9IHhfdHJhaW4sIA0KICB5ID0geF90cmFpbiwgDQogIGVwb2NocyA9IDIwMDAsDQogIHZlcmJvc2UgPSAwDQopDQoNCiMgZXZhbHVhdGUgdGhlIG1vZGVsDQpldmFsdWF0ZShtb2RlbDIsIHhfdHJhaW4sIHhfdHJhaW4pDQpgYGANCg0KSXQgc2VlbXMgdGhhdCBpdCBkb2VzIHNlcGVyYXRlIGJldHRlciB0aGFuIFBDQSBhYm92ZQ0KYGBge3J9DQppbnRlcm1lZGlhdGVfbGF5ZXJfbW9kZWwgPC0ga2VyYXNfbW9kZWwoaW5wdXRzID0gbW9kZWwyJGlucHV0LCBvdXRwdXRzID0gZ2V0X2xheWVyKG1vZGVsMiwgImJvdHRsZW5lY2siKSRvdXRwdXQpDQppbnRlcm1lZGlhdGVfb3V0cHV0IDwtIHByZWRpY3QoaW50ZXJtZWRpYXRlX2xheWVyX21vZGVsLCB4X3RyYWluKQ0KDQojIHBsb3QgdGhlIHJlZHVjZWQgZGF0IHNldA0KYWVkZiA8LSBkYXRhLmZyYW1lKG5vZGUxID0gaW50ZXJtZWRpYXRlX291dHB1dFssMV0sIG5vZGUyID0gaW50ZXJtZWRpYXRlX291dHB1dFssMl0sIG5vZGUzID0gaW50ZXJtZWRpYXRlX291dHB1dFssM10pDQphZV9wbG90bHkgPC0gcGxvdF9seShhZWRmLCB4ID0gfm5vZGUxLCB5ID0gfm5vZGUyLCB6ID0gfm5vZGUzLCBjb2xvciA9IH5haXMkc2V4KSAlPiUgYWRkX21hcmtlcnMoKQ0KDQphZV9wbG90bHkNCmBgYA0KDQojIyBSZWNvbnN0cnVjdGlvbiBFcnJvcjogUENBIHZzIEF1dG9FbmNvZGVyDQpgYGB7cn0NCiMgcENBIHJlY29uc3RydWN0aW9uDQpwY2EucmVjb24gPC0gZnVuY3Rpb24ocGNhLCB4LCBrKXsNCiAgbXUgPC0gbWF0cml4KHJlcChwY2EkY2VudGVyLCBucm93KHBjYSR4KSksIG5yb3cgPSBucm93KHBjYSR4KSwgYnlyb3cgPSBUKQ0KICByZWNvbiA8LSBwY2EkeFssMTprXSAlKiUgdChwY2Ekcm90YXRpb25bLDE6a10pICsgbXUNCiAgbXNlIDwtIG1lYW4oKHJlY29uIC0geCleMikNCiAgcmV0dXJuKGxpc3QoeCA9IHJlY29uLCBtc2UgPSBtc2UpKQ0KfQ0KDQp4aGF0IDwtIHJlcChOQSwgMTApDQpmb3IoayBpbiAxOjEwKXsNCiAgeGhhdFtrXSA8LSBwY2EucmVjb24ocGNhLCB4X3RyYWluLCBrKSRtc2UNCn0NCmBgYA0KDQoNCmBgYHtyfQ0KIyBBdXRvZW5jb2RlciByZWNvbnN0cnVjdGlvbg0KIyBhdXRvZW5jb2RlciBsYXN0IGxheWVyIGlzIHJlY29uc3RydWN0aW9uDQphZS5tc2UgPC0gcmVwKE5BLCA1KQ0KZm9yKGsgaW4gMTo1KXsNCiAgbW9kZWxrIDwtIGtlcmFzX21vZGVsX3NlcXVlbnRpYWwoKQ0KICBtb2RlbGsgJT4lDQogICAgbGF5ZXJfZGVuc2UodW5pdHMgPSA2LCBhY3RpdmF0aW9uID0gInRhbmgiLCBpbnB1dF9zaGFwZSA9IG5jb2woeF90cmFpbikpICU+JQ0KICAgIGxheWVyX2RlbnNlKHVuaXRzID0gaywgYWN0aXZhdGlvbiA9ICJ0YW5oIiwgbmFtZSA9ICJib3R0bGVuZWNrIikgJT4lDQogICAgbGF5ZXJfZGVuc2UodW5pdHMgPSA2LCBhY3RpdmF0aW9uID0gInRhbmgiKSAlPiUNCiAgICBsYXllcl9kZW5zZSh1bml0cyA9IG5jb2woeF90cmFpbikpDQoNCiAgbW9kZWxrICU+JSBjb21waWxlKA0KICAgIGxvc3MgPSAibWVhbl9zcXVhcmVkX2Vycm9yIiwgDQogICAgb3B0aW1pemVyID0gImFkYW0iDQogICkNCg0KICBtb2RlbGsgJT4lIGZpdCgNCiAgICB4ID0geF90cmFpbiwgDQogICAgeSA9IHhfdHJhaW4sIA0KICAgIGVwb2NocyA9IDUwMDAsDQogICAgdmVyYm9zZSA9IDANCiAgKQ0KDQogIGFlLm1zZVtrXSA8LSB1bm5hbWUoZXZhbHVhdGUobW9kZWxrLCB4X3RyYWluLCB4X3RyYWluKSkNCn0NCmBgYA0KDQpgYGB7cn0NCmRmIDwtIGRhdGEuZnJhbWUoayA9IGMoMToxMCwgMTo1KSwgbXNlID0gYyh4aGF0LCBhZS5tc2UpLCBtZXRob2QgPSBjKHJlcCgicGNhIiwgMTApLCByZXAoImF1dG9lbmNvZGVyIiwgNSkpKQ0KZ2dwbG90KGRmLCBhZXMoeCA9IGssIHkgPSBtc2UsIGNvbCA9IG1ldGhvZCkpICsgZ2VvbV9saW5lKCkNCmBgYA0KIyMjIyBPYnNlcnZhdGlvbg0KKiBXaGVuIGsgaXMgc21hbGwsIGF1dG9lbmNvZGVyIGhhcyBsZXNzIHJlY29uc3RydWN0aW9uIGVycm9yDQoqIFdoZW4gayBpcyBsYXJnZXIsIGF1dG9lbmNvZGVyIGFuZCBQQ0EgYXBlcGFyIHRvIGhhdmUgYWxtb3N0IHRoZSBzYW1lIHJlY29uc3RydWN0aW9uIGVycm9yDQoNCg0K